package indexer.processer.biquge

import indexer.processer.UrlPageProcessor
import org.slf4j.LoggerFactory
import us.codecraft.webmagic.Site
import us.codecraft.webmagic.selector.Selectable

/**
 * Created by dm on 2017/5/19 0019.
 */



class BiqugePageProcessor : UrlPageProcessor {
    override fun getUrl(): String = "http://www.biquzi.com"

    val Log = LoggerFactory.getLogger(javaClass)
    override fun getSite(): Site {
        return Site.me().setRetryTimes(3).setSleepTime(1000).setCycleRetryTimes(3).setTimeOut(60_000).setRetrySleepTime(10_000)
    }

    override fun process(page: us.codecraft.webmagic.Page) {
        val html = page.html

        val urls = html.links().regex("https?://.*\\.biquzi.com.*").all()
        page.addTargetRequests(urls)

        val mainInfo: Selectable = html.xpath("//div[@id='maininfo']")

        if (mainInfo.match()) {
            val name = html.xpath("//div[@id='info']/h1/text()").get()
            val author = html.xpath("//div[@id='info']/p[1]/text()").get()
            val url = page.request.url
            page.putField("name",name.trim())
            page.putField("url",url)
            page.putField("author",author)

        } else {
            page.setSkip(true)

        }


    }

}